rm(list=ls())
gc()

library(dplyr)
library(RSwissMaps)
library(rjson)
library(ggeffects)
library(rstan)
library(ggplot2)
library(arm)
library(lme4)
library(rjson)
library(pxR)
library(performance)
library(lmeresampler)
library(broom.mixed)

options(mc.cores = parallel::detectCores())

##### 1.a Load the models ####
load("model/Model party ideal position.rda")
load("model/Model party ideal position_by legislature.rda")

#### 1.b. Extract parameters ########

## Simple model
betas_mod1 <- summary(model1, pars = "beta")

betas_mod1 <- as.data.frame(betas_mod1)

## Dynamic model
betas_mod2 <- summary(model2, pars = "beta")

betas_mod2 <- as.data.frame(betas_mod2)

############### Get the population data #########

px_data <- read.px(file = url("https://dam-api.bfs.admin.ch/hub/api/dam/assets/32426125/master"), encoding = "UTF-8")
data_muni <- as.data.frame(px_data)
data_muni <- data_muni[data_muni$Ergebnis=="Ja in %",2:4]
colnames(data_muni) <- c("DateName", "Place", "YesShare")

data_json <- fromJSON(paste(readLines("https://www.pxweb.bfs.admin.ch/api/v1/fr/px-x-1703030000_101/px-x-1703030000_101.px"), collapse=""))
muni_id <- as.data.frame(list(number_mun = data_json[["variables"]][[1]][["values"]],
                              Place = levels(data_muni$Place)))

## loads the ids for projects

valid_items <- which(as.Date(substr(data_json[["variables"]][[2]][["valueTexts"]], 1, 10)) %in% as.Date(substr(levels(data_muni$DateName), 1, 10)))

project_id <- as.data.frame(list(anr = data_json[["variables"]][[2]][["values"]][valid_items],
                                 DateName = levels(data_muni$DateName)))

data_muni2 <- merge(data_muni, muni_id, by = "Place")
data_muni3 <- merge(data_muni2, project_id, by = "DateName")

data_muni3 <- data_muni3 %>% 
  mutate(place_type = ifelse(substr(Place, 1, 1)=="S", "Country",
                             ifelse(substr(Place, 1, 1)=="-", "Canton",
                                    ifelse(substr(Place, 1, 1)==">", "District",
                                           ifelse(substr(Place, 1, 1)==".", "Municipality", NA)))),
         id.place = ifelse(nchar(number_mun)<=4, number_mun,
                           ifelse(nchar(number_mun)>4, substr(number_mun, nchar(number_mun) - 3, nchar(number_mun)), NA)),
         year = as.numeric(substr(DateName, 1, 4)))

data_muni3$anr <- as.numeric(data_muni3$anr)/10
data_muni3$YesShare <- as.numeric(as.character(data_muni3$YesShare))


## Download the swissvotes data
swissvotes <- read.csv(url("https://swissvotes.ch/page/dataset/swissvotes_dataset.csv"), sep = ";")

## Select columns and observation
swissvotes <- swissvotes[swissvotes$anr>=193 & ### After 1965
                           swissvotes$anr<=664 & ###Before 2015 
                           swissvotes$rechtsform<=3, ## Only initivative, referenda mandatory and facultative
                         c(which(colnames(swissvotes)=="anr"),
                           which(colnames(swissvotes)=="datum"),
                           which(colnames(swissvotes)=="p.fdp"),
                           which(colnames(swissvotes)=="p.cvp"),
                           which(colnames(swissvotes)=="p.svp"),
                           which(colnames(swissvotes)=="p.gps"),
                           which(colnames(swissvotes)=="p.sps"),
                           which(colnames(swissvotes)=="rechtsform"),
                           which(colnames(swissvotes)=="volkja.proz"),
                           which(colnames(swissvotes)=="legislatur"))]

valid_ids <- unique(swissvotes$anr)

data_muni4 <- merge(data_muni3, swissvotes, by = "anr")



######## Prepare data for analyses for cantons and municipalities #######


## Select cases
data_canton <- data_muni4[data_muni4$place_type=="Canton",]

## Merge with betas
data_canton$i <- as.numeric(as.factor(data_canton$anr))
# plot(betas_mod1$c_summary.50..chain.1, betas_mod2$c_summary.50..chain.1)

###### Prepare the dependent variable ---> Difficulty canton - difficulty Swiss 

data_canton$CH_res <- rep(data_muni4[data_muni4$place_type=="Country",]$YesShare, each=26) ### Yes in percent at the national level

data_canton$YesShare <- data_canton$YesShare/100 #### Divide the percent by 100 to have a 0 to 1 indicator
data_canton$CH_res <- data_canton$CH_res/100

data_canton$log_CH_res <- log(data_canton$CH_res/(1-data_canton$CH_res)) #### Compute the inverse logit of this value
data_canton$log_YesShare <- log(data_canton$YesShare/(1-data_canton$YesShare)) 

data_canton$diff_log <- data_canton$log_YesShare - data_canton$log_CH_res ##### Compute the difference. 

data_canton$j <- as.numeric(as.factor(data_canton$id.place)) 

#### Merge with betas
betas_mod1$i <- 1:nrow(betas_mod1)
betas_mod2$i <- 1:nrow(betas_mod2)

data_canton_simple <- merge(data_canton, betas_mod1, by = "i")
data_canton_dynamic <- merge(data_canton, betas_mod2, by = "i")


######### For municipalitites ####

data_mun <- data_muni4[data_muni4$place_type=="Municipality",]

data_mun$i <- as.numeric(as.factor(data_mun$anr))

data_mun$CH_res <- rep(data_muni4[data_muni4$place_type=="Country",]$YesShare, each=length(unique(data_mun$Place)))

data_mun <- data_mun %>% 
  mutate(YesShare = ifelse(YesShare==0, 0.01,
                           ifelse(YesShare==100, 0.99, YesShare/100)))

#data_mun$YesShare <- data_mun$YesShare/100
data_mun$CH_res <- data_mun$CH_res/100

data_mun$log_CH_res <- log(data_mun$CH_res/(1-data_mun$CH_res))
data_mun$log_YesShare <- log(data_mun$YesShare/(1-data_mun$YesShare))

data_mun$diff_log <- data_mun$log_YesShare - data_mun$log_CH_res

data_mun$j <- as.numeric(as.factor(data_mun$id.place)) 


data_mun_simple <- merge(data_mun, betas_mod1, by = "i")

data_mun_dynamic <- merge(data_mun, betas_mod2, by = "i")

########## Save the data ##########

data_canton_mun <- list(data_canton_simple = data_canton_simple, 
                        data_canton_dynamic = data_canton_dynamic, 
                        data_mun_simple = data_mun_simple, 
                        data_mun_dynamic = data_mun_dynamic)

save(data_canton_mun, file = "data/Data model cantons and municipalities.Rda")




